package net;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.ConnectException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.entity.BufferedHttpEntity;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.CoreConnectionPNames;
import org.apache.http.util.EntityUtils;

/**
 * 
 * PageFetcher
 * 
 * @author Lijie Wang, wanglj07@sei.pku.edu.cn 2011-11-25
 */
public class PageFetcher {
	private static final int TIME_OUT = 5000;

	public static void main(String[] args) {
		 System.out.println(fetchURL("http://ting.baidu.com/album/7592397", 0));
		getCharset("adf<META http-equiv=\"Content-Type\"	\r\ncontent=\"text/html;	charset=utf-8\">adf");
	}

	public static String fetchURL(String url){
		return fetchURL(url, 5000);
	}
	/**
	 * Fetch content for a given url
	 * 
	 * @param url
	 *            legal URL, e.g.,
	 *            http://www.apiexample.com/clusterlist.html?classname=org.apache.commons.mail.Email
	 * @param time_out
	 *            0 with default time_out setting, 5000ms
	 * @return
	 */
	public static String fetchURL(String url, int time_out) {
		StringBuffer sb = new StringBuffer();
		HttpClient httpClient = new DefaultHttpClient();

		// Simulate Mozilla browser
		httpClient.getParams().setParameter("User-Agent",
				"Mozilla/5.0 (Windows NT 6.1; rv:8.0.1) Gecko/20100101 Firefox/8.0.1");
		// Timeout threshold
		httpClient.getParams().setParameter(CoreConnectionPNames.SO_TIMEOUT, time_out == 0 ? TIME_OUT : time_out);

		// Using HTTP GET method
		HttpGet httpGet = new HttpGet(url);
		try {
			// Execute the method to get response
			HttpResponse reponse = httpClient.execute(httpGet);
			// check status code of response
			int statusCode;
			StatusLine statusLine = reponse.getStatusLine();
			if (statusLine != null) {
				statusCode = statusLine.getStatusCode();
				System.err.println(statusCode);
				if (statusCode == HttpStatus.SC_OK) {
					// status code is ok, go on
					HttpEntity httpEntity = reponse.getEntity();
					if (httpEntity != null) {
						// We might need to read the response more than once, so
						// we use BufferedHttpEntity
						httpEntity = new BufferedHttpEntity(httpEntity);

						String mimeType = EntityUtils.getContentMimeType(httpEntity);
						if (mimeType != null && mimeType.toLowerCase().equals("text/html")) {
							// continue if the fetched page is a html webpage,
							// otherwise quit
							String charSet = EntityUtils.getContentCharSet(httpEntity);
							if (charSet != null) {
								// The charset is specified
								System.err.println("Charset:" + charSet);
								sb = readContent(httpEntity, charSet);
							} else {
								// The charset is not specified, here we need to do something complicated to detect the
								// charset correctly
								System.err.println("The charset is not specified!");
								// ---------Firstly read the content using default charset------------
								BufferedReader br = new BufferedReader(new InputStreamReader(httpEntity.getContent()));
								String line = null, charsetString = null;
								while ((line = br.readLine()) != null) {
									// Try to find the charset once a line of
									// content was read
									sb.append(line + "\r\n");
									charsetString = getCharset(sb.toString());
									if (charsetString != null) {
										// Found a charsetstring
										break;
									}
								}
								// =====================
								if (charsetString != null) {
									// Got the charset, read the content again
									sb = readContent(httpEntity, charsetString);
								}
								// We cannot find the charset from the page content if the charsetstring is null. Then
								// the page content read with default charset will be returned, i.e., sb gotten in
								// the while statement above
							}
						} else {
							System.err.println("The returned page is not a html webpage");
						}
					} else {
						System.err.println("The entity is null!");
					}
				} else {
					System.err.println("Status code is not SC_OK, " + statusCode);
				}
			}

		} catch (ClientProtocolException e) {
			sb = new StringBuffer();
			System.err.println(e.getMessage());
//			e.printStackTrace();
		} catch (IOException e) {
			sb = new StringBuffer();
			System.err.println(e.getMessage());
//			e.printStackTrace();
		} finally {
			// release all resources
			httpClient.getConnectionManager().shutdown();
		}
		return sb.toString();
	}

	/**
	 * Read content into StringBuffer from a HttpEntity with specified charset
	 * 
	 * @param httpEntity
	 * @param charsetString
	 * @return
	 * @throws UnsupportedEncodingException
	 * @throws IOException
	 */
	private static StringBuffer readContent(HttpEntity httpEntity, String charsetString)
			throws UnsupportedEncodingException, IOException {
		StringBuffer sb = new StringBuffer();
		BufferedReader br = null;
		String new_line = null;
		if (charsetString != null) {
			br = new BufferedReader(new InputStreamReader(httpEntity.getContent(), charsetString));
		} else {
			br = new BufferedReader(new InputStreamReader(httpEntity.getContent()));
		}
		while ((new_line = br.readLine()) != null) {
			sb.append(new_line + "\r\n");
		}
		return sb;
	}

	/**
	 * Get the charset specified in the page content: "<meta http-equiv="Content-Type"	content="text/html;
	 * charset=gb2312">" "<meta http-equiv="Content-Type" content="text/html; charset=gb2312" />"
	 * "    <meta http-equiv="Content-Type" content="text/html;charset=utf-8" />"
	 * 
	 * @param pageContent
	 * @return
	 */
	public static String getCharset(String pageContent) {
		if (pageContent == null || pageContent.trim().length() == 0) {
			return null;
		}
		Pattern pattern = Pattern.compile("<meta.*charset\\s*=([a-zA-Z\\d_-]+)\".*>", Pattern.DOTALL | Pattern.CASE_INSENSITIVE);
		Matcher matcher = pattern.matcher(pageContent);
		String charsetString = null;
		if (matcher.find()) {
			System.err.println("Found the charset!");
			charsetString = matcher.group(1);
			System.err.println(charsetString);
			return charsetString;
		}
		return null;
	}
}
